# -*- coding: utf-8 -*-
# File: ubbparser.py
# Date: 2011-11-7
# Author: xbyou

import re

#ftp, http, https, ssh, sftp
maybe_url = ['f', 'h', 's']
urls = ['ftp', 'http', 'https', 'ssh', 'sftp']
valid = ['-', '?', '.', '=', '&', '+', '%', '/', '#', ';', '@', ':', '~', '!', '`,', '_', ',', '(', ')']
invalids = ['"', "'", ' ', ']', ';']
div_invalids = ['"',"'",']',';']
types1 = ['bold', 'italic', 'code', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'color','paragraph','division','strike','em','br','super']#'url href' len(types1) = 11
len_types1 = len(types1) #非富文本格式
types = types1 + ['flash', 'image', 'url', 'div_url', 'video', 'ref', 'ol', 'ul' ,'dl']#'url href' len(types) = 19
len_types = len(types)
BEGIN = ['<b>', '<i>', '<code>', '<h1', '<h2', '<h3', '<h4', '<h5', '<h6', '<blockquote><div class="cmt-line"></div>', '''<span style="color:''','<p>','''<div''', '<s>','<em>', '<br/>', '<sup class="','''<embed src="''', '''<img src="''', '''<a href="''', '''<a href="''', '''<embed src="''', '''<!--# include virtual="/slice/common/article_item/''', '''<ol''', '''<ul''','''<dl''']
spe_urlend = '''" target="_blank">'''
spe_urlend_map = {'_blank':spe_urlend,'_self':'''" target="_self">'''}
spe_class = 'none'
spe_class_map = {'ol':'gui-ubbv-ol','ul':'gui-ubbv-ul'}
END = ['</b>', '</i>', '</code>', '</h1>', '</h2>', '</h3>', '</h4>', '</h5>', '</h6>', '''<div class="cmt-line"></div></blockquote>''', '</span>', '</p>' , '</div>' ,'</s>', '</em>' ,'','</sup>','''" type="application/x-shockwave-flash" allowscriptaccess="sameDomain" allowfullscreen="true" wmode="transparent" width="480" height="400"></embed>''', '''" />''', '</a>', '</a>', '''" type="application/x-shockwave-flash" allowscriptaccess="sameDomain" allowfullscreen="true" wmode="transparent" width="480" height="400"></embed>''', '''" -->''', '''</ol>''', '''</ul>''','''</dl>''']
# font, align, size, email
#? fly, glow, shadow, strike, sup, sub, light...etc
crange = lambda c1, c2: [chr(c) for c in range(ord(c1), ord(c2)+1)]
leaveclass = ['typeid', 'start', 'end', 'pair', 'parent', 'child', 'deep']
xss = ['javascript', 'vbscript', 'expression', 'applet', 'meta', 'xml', 'blink', 'link', 'style', 'script', 'embed', 'object', 'iframe', 'frame', 'frameset', 'ilayer', 'layer', 'bgsound', 'title', 'base', 'onabort', 'onactivate', 'onafterprint', 'onafterupdate', 'onbeforeactivate', 'onbeforecopy', 'onbeforecut', 'onbeforedeactivate', 'onbeforeeditfocus', 'onbeforepaste', 'onbeforeprint', 'onbeforeunload', 'onbeforeupdate', 'onblur', 'onbounce', 'oncellchange', 'onchange', 'onclick', 'oncontextmenu', 'oncontrolselect', 'oncopy', 'oncut', 'ondataavailable', 'ondatasetchanged', 'ondatasetcomplete', 'ondblclick', 'ondeactivate', 'ondrag', 'ondragend', 'ondragenter', 'ondragleave', 'ondragover', 'ondragstart', 'ondrop', 'onerror', 'onerrorupdate', 'onfilterchange', 'onfinish', 'onfocus', 'onfocusin', 'onfocusout', 'onhelp', 'onkeydown', 'onkeypress', 'onkeyup', 'onlayoutcomplete', 'onload', 'onlosecapture', 'onmousedown', 'onmouseenter', 'onmouseleave', 'onmousemove', 'onmouseout', 'onmouseover', 'onmouseup', 'onmousewheel', 'onmove', 'onmoveend', 'onmovestart', 'onpaste', 'onpropertychange', 'onreadystatechange', 'onreset', 'onresize', 'onresizeend', 'onresizestart', 'onrowenter', 'onrowexit', 'onrowsdelete', 'onrowsinserted', 'onscroll', 'onselect', 'onselectionchange', 'onselectstart', 'onstart', 'onstop', 'onsubmit', 'onunload']
color_name = ["red", "cyan", "blue", "darkblue", "lightpurple", "purple", "yellow", "lime", "fuchsia", "white", "silver", "grey", "black", "orange", "brown", "marsoon", "green", "olive"]
swf_urls = ['.youku.com', '.tudou.com', '.56.com', '.ku6.com', '.sohu.com', '.qq.com', '.sina.com', '.netease.com', '.126.net', '.qiyi.com']
words = crange('a', 'z') + crange('A', 'Z') + crange('0', '9')
TYPES_DICT = {}
for i,t in enumerate(types):
    TYPES_DICT[t] = i
MAX_REF_COUNT = 0
CLASS_PATTERN = re.compile('class=([\'|\"])?([\w|\x09|\x20|-]+)(\1)?')
TARGET_PATTERN = re.compile('target=([\'|\"])?(_blank|_self)(\1)?')
ID_PATTERN = re.compile('id=([\'|\"])?([\w|-]+)(\1)?')

class leave:

    def __init__(self, typeid, start, ed, pair, parent, deep, show=1, styleclass='',target='_blank',lid = ''):
        self.typeid = typeid
        self.start = start
        self.end = ed
        self.pair = pair
        self.parent = parent
        self.deep = deep
        self.show = show
        #如果指定styleclass="none"，标记的class为空
        #如果指定styleclass="",标记的class为spe_class_map的默认值，如果没有默认值那就是空
        #其它情况下class=styleclass
        self.styleclass = styleclass
        if not self.styleclass:
            clazz = spe_class_map.get(types[self.typeid],'')
            if clazz:
                self.styleclass = clazz
        elif self.styleclass == spe_class:
            self.styleclass = ''
        self.target = target
        self.lid = lid

    def __repr__(self):
        s = []
        s.append('type:' + types[self.typeid])
        s.append('start:' + str(self.start))
        s.append('end:' + str(self.end))
        s.append('pair:' + str(self.pair))
        s.append('parent:' + str(self.parent))
        s.append('deep:' + str(self.deep))
        s.append('show:' + str(self.show))
        s.append('class:' + str(self.styleclass))
        s.append('target:' + str(self.target))
        s.append('id:' + str(self.lid))
        return '\n{' + ';'.join(s) + '}'

class buildtree:
    #treelist (type, start, end, pair, parent, children, deep
    #return -1 means not SUCCESS

    def __init__(self, text, ref_count=MAX_REF_COUNT):
#        self.text = unicode(text, "utf-8")
        if type(text) != unicode:
            self.text = text.decode('u8')
        else:
            self.text = text
        self.length = len(self.text)
        self.step = 0
        self.ref_count = ref_count
        self.tree = []

    def nxt(self):
        self.step += 1
        return True if self.step + 1 < self.length else False

    def url(self):
    #analysis url in text,out of [] or [/]
        div_url_id = TYPES_DICT['div_url']
        if self.step + 8 > self.length:
            self.step += 1
            return       #It can't be url
        isurl = False
        for ur in urls:
            if self.text[self.step:self.step + len(ur) + 3] == ur + '://':
                urllength = len(ur) + 3
                isurl = True
                break
        if not isurl:
            self.step += 1 
            return       #It is not a url
        loc = self.step + urllength - 1
        while loc < self.length - 1:
            if self.text[loc + 1] in words + valid:
                loc += 1
            else:
                break
        # -1 means no pair
        f = self.parent(div_url_id)
        l = leave(div_url_id, self.step, loc, len(self.tree), f, -1, show=1)
        self.tree.append(l)
        self.step = loc
        if not self.nxt(): return "It's End"

    def deep(self):
        for i, l in enumerate(self.tree):
            if l.parent == -1:
                self.tree[i].deep = 0
            else:
                self.tree[i].deep = self.tree[self.tree[i].parent].deep + 1

    def match(self):
        #len(pos) == 0 fail; len(pos) == 1 bingo
        pos = []
        for i, t in enumerate(types):
            if t[0] == self.text[self.step].lower():
                pos.append(i)
        loc = self.step + 1
        while len(pos) > 0 and loc < self.length:
            found = []
            for t in pos:
                if types[t][loc - self.step] == self.text[loc].lower():
                    found.append(t)
            pos = found
            found = []
            loc = loc + 1
            if len(pos) == 1 and loc - self.step == len(types[pos[0]]): break
        if len(pos) == 1:
            self.step = loc - 1
            #指定定位后的标签紧跟的一位，放弃
            return pos[0]
        self.step += 1
        return "False Match"

    def trans(self, mode):
        pass

    def pair(self, t):
        f = len(self.tree) - 1
        if f == -1: return -2
        while f >= 0 and (self.tree[f].typeid != t or self.tree[f].pair >= 0):
            f -= 1
        if f >= 0:
            self.tree[f].pair = len(self.tree)
            return f
        else:
            return -2

    #字体变化不会做parent 他们没有子孙！！！so 重构parent: 完成
    #checkpair后parent和children的情况会变化 之前居然忘处理了= =# 完成
    def parent(self, t):
        if len(self.tree) == 0: return -1
        if t < len_types1: return -1
        f = len(self.tree) - 1
        while self.tree[f].typeid < len_types1 and f >= 0: f -= 1
        if self.tree[f].pair >= 0:
            return self.tree[f].parent
        else:
            return f

        #暂时看只有img可以修饰url，其他情况不允许嵌套
        #这里不检查嵌套错误，因为可能存在假的[]

    def findend(self, t):
        t_d = TYPES_DICT
        if t< len_types1 and t not in(t_d['color'],t_d['division'],t_d['super'],t_d['h1'],t_d['h2'],t_d['h3'],t_d['h4'],t_d['h5'],t_d['h6']):
            if self.step + 1 >= self.length: return -1
            if self.text[self.step + 1] == ']':
                return self.step + 1
            else:
                return -1
        #这里写的不好 index需要替换#TODO
        ed = self.text.find(']', self.step)
        if ed == -1:
            return -1
        else:
            err = self.text.find('[', self.step, ed)
            if err == -1:
                return ed
            else:
                return -1

    def parser(self):
        #-2 means that it haven't a pair yet
        brother = False
        p = -2
        f = -1
        start = self.step
        if not self.nxt(): return -1
        if self.text[self.step] == '/':
            brother = True
            if not self.nxt(): return -1
        t = self.match()
        if t == "False Match": 
            self.step = start + 1
            return -1
        loc = self.findend(t)
        if loc == -1: 
            self.step = start + 1
            return -1
        if brother:
            p = self.pair(t)
            if p != -2:
                f = self.tree[p].parent
        else:
            f = self.parent(t)
        if p != -1:
            styleclass = '' if brother else self.findclass(loc)
            url_target = self.findtarget(loc) if t == TYPES_DICT['url'] else '_blank'
            l_id = self.findid(loc)
            l = leave(t, start, loc, p, f, -1, 1, styleclass,url_target,l_id)
            self.step = loc
            self.tree.append(l)
            if not self.nxt(): return -1

    def findclass(self,locend):
        attrs = self.text[self.step:locend]
        m = CLASS_PATTERN.search(attrs)
        if not m:
            return ''
        l,s,l = m.groups()
        if l:
            return s
        ss = s.split()
        if ss:
            return ss[0]

    def findtarget(self,locend):
        attrs = self.text[self.step:locend]
        m = TARGET_PATTERN.search(attrs)
        if not m:
            return '_blank'
        l,s,l = m.groups()
        if l:
            return s
        ss = s.split()
        if ss:
            return ss[0]

    def findid(self,locend):
        attrs = self.text[self.step:locend]
        m = ID_PATTERN.search(attrs)
        if not m:
            return ''
        l,s,l = m.groups()
        return s

    def determine(self):
        ch = self.text[self.step]
        if ch == '[':
            self.parser()
        elif ch in maybe_url:
            self.url()
        else:
            self.step += 1

    #关于相邻标签对调等一些情况
    def swap(self):
        pass

    def checkpair(self):
        #检查配对，保证配对的正确性和完整性，尚需完善#TODO
        """仅允许image和url的嵌套"""
#        ref_count = 0
        length_tree = len(self.tree)
        for i in xrange(length_tree):
            l = self.tree[i]
            l_parent = l.parent
            while not self.tree[l_parent].show:
                l_parent = self.tree[l_parent].parent
            if l.pair == -2 \
               or l_parent != -1 and not (l.typeid == TYPES_DICT['image'] and self.tree[l.parent].typeid == TYPES_DICT['url'] or self.tree[l.parent].typeid in (TYPES_DICT['ul'], TYPES_DICT['ol'], TYPES_DICT['dl'])):
#               or l.typeid == TYPES_DICT['ref'] and l.pair > i and ref_count >= self.ref_count:
                if l.typeid != TYPES_DICT['div_url'] and l.pair != -2:
                    self.tree[l.pair].pair = -2
                l.show = 0
#            if l.typeid == TYPES_DICT['ref'] and l.pair > i:
#                ref_count += 1
        #ref_check放到compiler以后，否则无法确认是不错误配对还是因为数量限制的原因

    def create(self):
        while self.step < self.length:
            self.determine()
        self.checkpair()
        self.swap()
        self.deep()
        return self.tree

def xss_filter(s):
#    result = ''
#    for ch in s:
#        try:
#            c = urllib.quote_plus(ch)
#        except:
#            result += ch
#        else:
#            result += c
#    return result
#        return s
    return s.replace('"', '%22').replace("'", '%27').replace('=', '%3D').replace('(', '%28').replace(')', '%29')

def while_valid(u, u_valids):
    result = ''
    pos = 0
    u_length = len(u)
    while pos < u_length and u[pos] in u_valids:
        result += u[pos]
        pos += 1
    return u[:pos]

def html_escape(s, sp=False):
    ret = s.replace('<', '&lt;').replace('>', '&gt;')
    return ret.replace(' ', '&nbsp;') if sp else ret

def swf(u):
    if not '.swf' in u:
        swf_valid = words + ['_'] + ['='] + ['-'] 
        if u[:29] == 'http://v.youku.com/v_show/id_':
            return 'http://player.youku.com/player.php/sid/' + while_valid(u[29:], swf_valid) + '/v.swf'
        elif u[:35] == 'http://www.tudou.com/programs/view/':
            return 'http://www.tudou.com/v/' + while_valid(u[35:], swf_valid) + '/v.swf'
        elif u[:22] == 'http://v.ku6.com/show/':
            return 'http://player.ku6.com/refer/' + while_valid(u[22:], swf_valid) + '/v.swf'
        elif u[:30] == 'http://v.ku6.com/special/show_':
            loc = u.find('/', 30) + 1
            return 'http://player.ku6.com/refer/' + while_valid(u[loc:], swf_valid) + '/v.swf'
        elif u[:18] == 'http://www.56.com/':
            loc = u.find('/', 18) + 1
            if loc != -1 and u[loc:loc + 2] == 'v_':
                return 'http://player.56.com/v_' + while_valid(u[loc + 2:], swf_valid) + '.swf'
#        elif u[:15] == 'http://v.qq.com':
#            parts = u.split('/')
#            for p in parts:
#                if '.html' in p:
#                    v_id = p[:-5]
#            return 'http://cache.tv.qq.com/qqplayerout.swf?vid=%s' % v_id
    else:
        for swf_url in swf_urls:
            if swf_url in u:
                return u
    return -1

class RefError(Exception):
    pass

class RefCountError(RefError):
    """ref标签数量超出规定max_ref_count"""
    pass

class RefContentError(RefError):
    """ref标签内容错误，非果壳网链接"""
    pass

re_ref_content = re.compile("(http://)?www.guokr.com/(zone/[^/]*/)?(post|article|blog|question)/\d+(/)?")

class compiler:

    def __init__(self, text, ref_count=MAX_REF_COUNT, space=False):
        self.build = buildtree(text, ref_count)
        self.text = self.build.text
        self.tree = self.build.create()
        self.ref_count = ref_count
        self.space = space
        self.part = []
        self.left = []
        self._mark = [False] * len(self.tree)
        self._depart()
        try:
            self.ref_check()
        except:
            pass
        self.DELETED = [] 
        self.TRANS = [i for i in xrange(len_types)]

    def _depart(self):
        length = len(self.tree)
        for i,l in enumerate(self.tree):
            p = self.text[l.start:l.end + 1]
            if '[ref]' in p or '[/ref]' in p:
                self.part.append('')
            else:
                self.part.append(p)
            if i == length - 1:
                end = len(self.text)
            else:
                end = self.tree[i + 1].start
            end = self.text[l.end + 1:end]
            if l.typeid in (TYPES_DICT['ul'], TYPES_DICT['ol'], TYPES_DICT['dl']) and l.show and l.pair < i:
                loc = end.index(end.lstrip())
                end = end[:loc].replace('\n', '', 1)+end[loc:]
            self.left.append(end)
    
    def _ref_content_check(self, text):
        text = text.replace(' ', '')
        if re_ref_content.match(text):
            return True
        else:
            return False

    def ref_check(self):
        ref_count = 0
        for i, l in enumerate(self.tree):
            if l.typeid == TYPES_DICT['ref'] and l.pair > i and l.show:
                if l.pair == i+1:
                    check_content = self.left[i]
                else:
                    check_content = self.left[i] + ''.join([self.part[j]+self.left[j] for j in xrange(i+1, l.pair)])

                ref_count += 1
                if ref_count > self.ref_count:
                    l.show = 0
                    self.tree[l.pair].show = 0
                    if self.tree[i+1].typeid == TYPES_DICT['div_url']:
                        self.tree[i+1].show = 1 ##TODO 可能存在风险
                if not self._ref_content_check(check_content):
                    #return self.LogicError('ContentError', u"ref标签内容有问题")
#                    raise RefError('ContentError', u"ref标签内容有问题")
                    raise RefContentError()
        if ref_count <= self.ref_count:
            return True
        else:
            #return self.LogicError('CountError', u"ref标签数量有问题")
            #raise RefError('CountError', u"ref标签数量有问题")
            raise RefCountError()

    def video(self, u):
        pass

    def trans(self, l, front, mid, back, whitelist):
        #现存问题是对标签内的内容分析不足，情况很多，需要有对应的list：关于一些不规范和事件
        t_d = TYPES_DICT
        t = l.typeid
        styleclass = l.styleclass
        url_target = spe_urlend_map[l.target]
        if t not in whitelist:
            return front + mid + back
        if t < len_types1:
            if mid[1] != '/':
                if t == t_d['color']:
                    mid = mid.replace('"', '').replace("'", '')
                    loc = mid.find('=')
                    if loc == -1:
                        mid = ''
                    else:
                        mid = mid[loc + 1:]
                        mid_length = len(mid)
                        loc = 0
                        while loc < mid_length:
                            if mid[loc] not in invalids:
                                loc += 1
                            else:
                                break
                        mid = mid[:loc]
                    return BEGIN[t] + mid + ''';">'''
                elif t == t_d['division']:
                    id_s = (' id="' + l.lid + '" ') if l.lid else ''
                    clazz_s = ' class="' + styleclass + '" ' if styleclass else ''
                    return BEGIN[t] + id_s + clazz_s + '''">'''
                elif t == t_d['super']:
                    mid = mid.replace('"', '').replace("'", '')
                    loc = mid.find('=')
                    if loc == -1:
                        mid = ''
                    else:
                        mid = mid[loc + 1:]
                        mid_length = len(mid)
                        loc = 0
                        while loc < mid_length:
                            if mid[loc] not in div_invalids:
                                loc += 1
                            else:
                                break
                        mid = mid[:loc]
                    return BEGIN[t] + mid + '''">'''
                elif t in (t_d['h1'],t_d['h2'],t_d['h3'],t_d['h4'],t_d['h5'],t_d['h6']):
                    id_s = (' id="' + l.lid + '" ') if l.lid else ''
                    return BEGIN[t] + id_s + '>'
                else:
                    return BEGIN[t]
            else:
                return END[t]
        elif t == t_d['div_url'] or t == t_d['url'] and len(front) == 5: #len('[url]') = 5 #单独的url
            return BEGIN[t] + mid + url_target + mid + END[t]
        elif t == t_d['url']:#url href
            #TODO 过滤除链接外的
            front = front.replace('"', '').replace("'", '')
            loc = front.find('=')#只找'=',以匹配各种形式的url标签
            if loc == -1:
                front = ''
            else:
                front = front[loc + 1:-1]
                front_length = len(front)
                loc = 0
                while loc < front_length:
#                    if front[loc] in words + valid:
                    if front[loc] not in invalids:
                        loc += 1
                    else:
                        break
                front = front[:loc]
            return BEGIN[t] + front + url_target + mid + END[t]
        elif t == t_d['video']: #video
            s = swf(mid)
            if s != -1:
                return BEGIN[t] + s + END[t]
            else:
                return front + mid + back
        elif t in (t_d['ol'], t_d['ul']):
            clazz = (' class="%s"' % styleclass) if styleclass else ''
            mid = '\n'.join([p.strip() if '<' in mid else xss_filter(p.strip()) for p in mid.split('\n') if p.strip()])
            return BEGIN[t] + clazz + '>' +'<li>' + mid.replace('\n', '</li><li>') + '</li>' + END[t]
        elif t == t_d['dl']:
            clazz = (' class="%s"' % styleclass) if styleclass else ''
            midlist = [p.strip() if '<' in mid else xss_filter(p.strip()) for p in mid.split('\n') if p.strip()]
            mid_dt = '<dt>' + (midlist[0] if midlist else '') + '</dt>'
            mid_dd = '\n'.join(map(lambda x:'<li>' + x + '</li>',midlist[1:] if midlist else []))
            return BEGIN[t] + clazz + '>'  + mid_dt + mid_dd + END[t]
        elif t == t_d['ref']:
            item_id = '-1'
            item = 'article'
            mid = mid.split('/')
            for s in mid:
                if s.isdigit(): 
                    item_id = s
                if s in ('article', 'post', 'blog', 'question'):
                    item = s
            if 'zone' in mid:
                item = 'zone'
            return BEGIN[t] + item_id + '/ref_' + item + '/' + END[t]
        else:
            return BEGIN[t] + xss_filter(mid) + END[t]

    def combine(self, start, end, whitelist):
        step = start
        ans = ''
        while step <= end:
            if (not self._mark[step]) and self.tree[step].show:
                l = self.tree[step]
                if l.typeid < len_types1 or l.typeid == TYPES_DICT['div_url']:
                    ans = ans + self.trans(l,'', self.part[step], '', whitelist) + html_escape(self.left[step], self.space)
                    self._mark[step] = True
                    step += 1
                else:
                    if l.pair == step + 1:
                        ans = ans + self.trans(l, self.part[step], self.left[step], self.part[step + 1], whitelist) + html_escape(self.left[step + 1], self.space)#链接等标签中无嵌套则亦无文字处理，即相邻
                        step += 1
                        self._mark[step] = True
                        self._mark[l.pair] = True
                    else:
                        l_next = self.tree[step + 1]
                        if l_next.typeid < len_types1:
                            mid = step + 1
                        else:
                            mid = l_next.pair
                        ans = ans + self.trans(l, self.part[step], self.left[step] + self.combine(step + 1, l.pair - 1, whitelist), self.part[l.pair], whitelist) + html_escape(self.left[l.pair], self.space) #TODO 需要确认一下
                        self._mark[step] = True
                        self._mark[l.pair] = True
                        step = l.pair + 1
            elif (not self._mark[step]) and (not self.tree[step].show):
                ans += html_escape(self.part[step] + self.left[step], self.space)
                self._mark[step] = True
                step += 1
            else:
                step += 1
        return ans

    def ubbtohtml(self, whitelist=['all']):
        if whitelist == ['all']:
            whitelist = self.TRANS
        else:
            whitelist = [TYPES_DICT[t] for t in whitelist]
        if len(self.tree) > 0:
            front = self.text[:self.tree[0].start]
        else:
            front = self.text
        return html_escape(front, self.space) + self.combine(0, len(self.tree) - 1, whitelist)

    def killubb(self, blacklist=[], at=False):
        for t in blacklist:
            try:
                type_id = TYPES_DICT[t]
            except:
                raise 'type %s is not in the tag list' % t
            else:
                self.DELETED.append(type_id)
        length_tree = len(self.tree)
        pos = 0

        if length_tree > 0:
            text = self.text[:self.tree[0].start]
        else:
            text = self.text
        while pos < length_tree:
            if at and self.tree[pos].typeid == TYPES_DICT['url'] and self.tree[pos].pair == pos + 1 and u'@' in self.left[pos]:
                text += self.part[pos] + self.left[pos] + self.part[pos+1] + self.left[pos+1]
                pos += 2
            elif self.tree[pos].typeid in self.DELETED and self.tree[pos].pair > pos and (self.tree[pos].show or self.tree[pos].typeid == TYPES_DICT['ref']):
                pos = self.tree[pos].pair
            elif not self.tree[pos].show and self.tree[pos].typeid != TYPES_DICT['ref'] or self.tree[pos].typeid == TYPES_DICT['div_url']:
                text += self.part[pos] + self.left[pos]
                pos += 1
            else:
                text += self.left[pos]
                pos += 1
        return html_escape(text, self.space)
    
    def spe_ul(self, comb, content = '', ):
        tmp = []
        for pt in comb:
            if type(pt[0]) == unicode:
                pa = False
            else:
                pa = True
            if '\n' in pt[0]:
                ps = pt[0].split('\n')
                l_ps = len(ps)
                for i, p in enumerate(ps):
                    if content and p.strip() != '':
                        tmp.append((p.strip() if l_ps-1 == i and p.strip() != '' else p.strip()+'\n', None))
                    else:
                        tmp.append((p.strip() if l_ps-1 == i and p.strip() != '' else p.strip()+'\n', pt[1]))
            elif pa or pt[0].strip() != '':
                tmp.append(pt)
        comb = tmp
        tmp = []
        t = []
        for pt in comb:
            if type(pt[0]) == unicode:
                pa = False
            else:
                pa = True
            if '\n' in pt[0]:
                p = pt[0].strip() if not pa else pt[0]
                pt = (p, pt[1])
                t = t + [pt] if p != '' else t
                if t:
                    tmp.append(tuple(t))
                t = []
            else:
                p = pt[0].strip() if not pa else pt[0]
                pt = (p, pt[1])
                if p:
                    t.append(pt)
        if t:
            if tmp:
                tmp.append(tuple(t))
            else:
                tmp = tuple(t)
        tmp = tuple(tmp)
        return (tmp, content) if content != '' else tmp

    def mark_build(self, start, end):
        ret = []
        loc = start
        while loc <= end:
            if self.tree[loc].show and self.tree[loc].pair > loc:
                loc_end = self.tree[loc].pair
                if loc_end == loc + 1:
                    comb = (self.left[loc], types[self.tree[loc].typeid])
                    if comb[1] in ('ul', 'ol', 'dl'):
                        comb = self.spe_ul([comb], comb[1])
                    ret.append(comb)
                    loc += 1
                elif loc_end == loc + 2 and self.tree[loc+1].typeid == TYPES_DICT['div_url'] and not self.tree[loc+1].show:
                    comb = (self.left[loc]+self.part[loc+1]+self.left[loc+1], types[self.tree[loc].typeid])
                    ret.append(comb)
                    loc = loc_end
                elif loc_end == loc + 2 and self.tree[loc+1].typeid == TYPES_DICT['div_url'] and self.tree[loc+1].show:
                    comb = [(self.left[loc], None), (self.part[loc+1], 'url')] if self.left[loc] else [(self.part[loc+1], 'url')]
#                    if len(comb) == 1:
#                        comb = comb[0]
                    ret.append((tuple(comb), types[self.tree[loc].typeid]))
                    loc = loc_end
                else:
                    comb = [(self.left[loc], None)] + self.mark_build(loc+1, loc_end-1) if self.left[loc] else self.mark_build(loc+1, loc_end-1)
                    if types[self.tree[loc].typeid] in ('ul', 'ol' , 'dl'):
                        comb = self.spe_ul(comb)
                    ret.append((tuple(comb), types[self.tree[loc].typeid]))
                    loc = loc_end
            elif self.tree[loc].show and self.tree[loc].pair == loc:  # self.tree[loc].typeid = TYPES_DICT['div_url']
#                ret.append(content_leave('url', self.part[loc]))
                ret.append((self.part[loc], 'url'))
                ret.append((self.left[loc], None)) if self.left[loc] != '' else ''
#                ret.append((self.left[loc], None)) if self.left[loc].strip() != '' else ''
                loc += 1
            elif self.left[loc]:
                ret.append((self.left[loc], None)) if self.left[loc] != '' else ''
#                ret.append((self.left[loc], None)) if self.left[loc].strip() != '' else ''
                loc += 1
            else:
                loc += 1
        return ret

    def pickstrip(self, tup):
        ret = []
        if not tup:
            return tup
        if type(tup[0]) == unicode:
            if tup[0].strip() != '':
                return tup
        for t in tup:
            if type(t[0]) == unicode:
                if t[0].strip() != '':
                    ret.append(t)
            else:
                if len(t) > 1:
                    ret.append((self.pickstrip(t[0]), t[1]))
                else:
                    ret.append((self.pickstrip(t[0]), ))
        return tuple(ret)

    def mark(self):
        #for soddy
        length = len(self.tree)
        if len(self.tree) > 0:
            front = self.text[:self.tree[0].start]
        else:
            front = self.text
        front = [(front, None)] if front != '' else []
        if length>0:
            ans = self.mark_build(0, length-1)
        else:
            ans = []
        ret = front + ans if ans else front
        return ret


###################################Unit Test
import unittest
class TestAll(unittest.TestCase):

    def test_bb2html_valid1(self):
        raw = """\
坑[bold]爹[/BOLD]呢[italic]阿[/italic]!
hello[image]url1[/image]!!
FF
url_s[url]url1[/url]xxx
url_t[url href=url1]fuck[/url]
[blockquote]oh,shit![/blockquote]
[url]http://guokr.com[/url]
"""

        wanted=u"""坑<b>爹</b>呢<i>阿</i>!
hello<img src="url1" />!!
FF
url_s<a href="url1" target="_blank">url1</a>xxx
url_t<a href="url1" target="_blank">fuck</a>
<blockquote><div class="cmt-line"></div>oh,shit!<div class="cmt-line"></div></blockquote>
<a href="http://guokr.com" target="_blank">http://guokr.com</a>
"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """\
坑[bold]爹[/BOLD]呢[italic]阿[/italic]!
hello[image]url1[/image]!!
"""

        wanted = u"""\
坑[bold]爹[/BOLD]呢[italic]阿[/italic]!
hello<img src="url1" />!!
"""

        c = compiler(raw)
        result = c.ubbtohtml(['image'])
        self.assertEqual(result, wanted)

        raw = """\
坑[bold]爹[/BOLD]呢[italic]阿[/italic]!
hello[image]url1[/image]!!
"""

        wanted = u"""\
坑<b>爹</b>呢<i>阿</i>!
hello[image]url1[/image]!!
"""

        c = compiler(raw)
        result = c.ubbtohtml(['bold', 'italic'])
        self.assertEqual(result, wanted)


    def test_span1(self):
        raw = """[color color="red" onclick="javascript:alert(\"hello\");"]anlong[/color]"""
        wanted = u"""<span style="color:red;">anlong</span>"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

    def test_span2(self):
        raw = """[color=red]wocao[/color]"""
        wanted = u"""<span style="color:red;">wocao</span>"""
        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)


#TODO 曾经出错的结果 2,3,4

    def test_bb2html_valid2(self):
        raw = """\
[bold]这玩儿坑爹呢
"""

        wanted = u"""\
[bold]这玩儿坑爹呢
"""
        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)


    def test_bb2html_valid3(self):
        raw = """\
[bold]这玩儿坑爹呢
[italic]这玩儿真的坑爹[/bold]123456[/italic]
"""

        wanted = u"""\
<b>这玩儿坑爹呢
<i>这玩儿真的坑爹</b>123456</i>
"""
        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

    def test_bb2html_valid4(self):
        raw = """\
[bold]坑[bold]爹呢[italic]阿!
hello[image]url1[/image]!!
FF
url_s[url]url1[/url]xxx
url_t[url href=url1][image]fuck[/image][/url]
[blockquote]oh,shit![/blockquote][/BOLD]
[url]http://guokr.com[/url][/italic]
"""
        wanted = u"""[bold]坑<b>爹呢<i>阿!
hello<img src="url1" />!!
FF
url_s<a href="url1" target="_blank">url1</a>xxx
url_t<a href="url1" target="_blank"><img src="fuck" /></a>
<blockquote><div class="cmt-line"></div>oh,shit!<div class="cmt-line"></div></blockquote></b>
<a href="http://guokr.com" target="_blank">http://guokr.com</a></i>
"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

    def test_performance(self):
        raw = """\
坑[bold]爹[/BOLD]呢[italic]阿[/italic]!
hello[image]url1[/image]!!
FF
url_s[url]url1[/url]xxx
url_t[url href=url1]fuck[/url]
[blockquote]oh,shit![/blockquote]
[url]http://guokr.com[/url]
""" * 1

        wanted = u"""坑<b>爹</b>呢<i>阿</i>!
hello<img src="url1" />!!
FF
url_s<a href="url1" target="_blank">url1</a>xxx
url_t<a href="url1" target="_blank">fuck</a>
<blockquote><div class="cmt-line"></div>oh,shit!<div class="cmt-line"></div></blockquote>
<a href="http://guokr.com" target="_blank">http://guokr.com</a>
""" * 1

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

    def test_bb2text_valid(self):
        raw = """\
[bold]abc[bold]坑[bold]爹[/BOLD]呢[italic]阿[/italic]!
hello[image]url1[/image]!!
FF
url_s[url]url1[/url]xxx
url_t[url href=url1]fuck[/url]
123[url=http://w.ca.lei]danteng[/ur]456
[blockquote]oh,shit![/blockquote]
"""
        wanted=u"""[bold]abc[bold]坑爹呢阿!
hellourl1!!
FF
url_surl1xxx
url_tfuck
123[url=http://w.ca.lei]danteng[/ur]456

""" #blockquote is in DELETED
        c = compiler(raw)
        result = c.killubb(['blockquote'])
#        result = c.killubb()
        self.assertEqual(result, wanted)

        raw = """[blockquote]...[/blockquote]"""
        wanted = u"""..."""
        c = compiler(raw)
        result = c.killubb()
        self.assertEqual(result, wanted)

        test_str = """[url href="http://www.guokr.com/n/果壳网烧饭阿姨/"]@果壳网烧饭阿姨[/url] 的鱼很不错，老师我很喜欢！"""
        wanted_str=u"""@果壳网烧饭阿姨 的鱼很不错，老师我很喜欢！"""
        result = compiler(test_str).killubb()
        self.assertEqual(result, wanted_str)

        c = compiler(raw)
        result = c.killubb()
        self.assertEqual(result, wanted)

        test_str = """http://www.guokr.com/n/果壳网烧饭阿姨/"""
        wanted_str=u"""http://www.guokr.com/n/果壳网烧饭阿姨/"""
        result = compiler(test_str).killubb()
        self.assertEqual(result, wanted_str)

        c = compiler(raw)
        result = c.killubb()
        self.assertEqual(result, wanted)

    def test_link_auto_check(self):
    #MASTER的测试数据中，第一行中href后有""会导致输出额外多"" 需要做出判断
        raw = """\
[url href="http://g.cn/"]狗狗[/url]
g.cn
http://g.cn
http://g.cn/
https://g.cn/
"""

        wanted = u"""<a href="http://g.cn/" target="_blank">狗狗</a>
g.cn
<a href="http://g.cn" target="_blank">http://g.cn</a>
<a href="http://g.cn/" target="_blank">http://g.cn/</a>
<a href="https://g.cn/" target="_blank">https://g.cn/</a>
"""
        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

    def test_youku(self):
        ## 正常结果
        u = '''http://v.youku.com/v_show/id_XMzEzNzY3NDY0.html'''
        result = swf(u)
        wanted = '''http://player.youku.com/player.php/sid/XMzEzNzY3NDY0/v.swf'''

        self.assertEqual(result, wanted)

        ## 失败结果
        u = '''http://www.youku.com'''
        result = swf(u)

        self.assertEqual(result, -1)
        pass

    def test_tudou(self):
        ## 正常结果
        u = '''http://www.tudou.com/programs/view/sP87gstesX4/'''
        result = swf(u)
        wanted = '''http://www.tudou.com/v/sP87gstesX4/v.swf'''

        self.assertEqual(result, wanted)

        ## 失败结果
        u = '''http://www.tudou.com'''
        result = swf(u)

        self.assertEqual(result, -1)
        pass

    def test_ku6(self):
        ## 正常结果
        u = '''http://v.ku6.com/show/DC_d87X46c2wC2C4.html'''
        result = swf(u)
        wanted = '''http://player.ku6.com/refer/DC_d87X46c2wC2C4/v.swf'''

        self.assertEqual(result, wanted)

        u = '''http://v.ku6.com/special/show_6120043/ApGL6UVWp2ln4bj9.html'''
        result = swf(u)
        wanted = '''http://player.ku6.com/refer/ApGL6UVWp2ln4bj9/v.swf'''

        self.assertEqual(result, wanted)

        ## 失败结果
        u = '''http://www.ku6.com'''
        result = swf(u)

        self.assertEqual(result, -1)
        pass

    def test_56(self):
        ## 正常结果
        u = '''http://www.56.com/u27/v_NjM5Nzg2NTY.html'''
        result = swf(u)
        wanted = '''http://player.56.com/v_NjM5Nzg2NTY.swf'''

        self.assertEqual(result, wanted)

        ## 失败结果
        u = '''http://www.56.com'''
        result = swf(u)

        self.assertEqual(result, -1)
        pass
    
    def _test_qq(self): #弃用
        ## 正常结果
        u = '''http://v.qq.com/play/92gdWsyoafx.html'''
        result = swf(u)
        wanted = '''http://cache.tv.qq.com/qqplayerout.swf?vid=92gdWsyoafx'''

        self.assertEqual(result, wanted)

        ## 失败结果
        u = '''http://www.qq.com'''
        result = swf(u)

        self.assertEqual(result, -1)
        pass

    def test_valid(self):
        u = '''http://v.ku6.com/show/pdR2iisl-DDqDR-u.html'''
        result = swf(u)
        wanted = '''http://player.ku6.com/refer/pdR2iisl-DDqDR-u/v.swf'''

        self.assertEqual(result, wanted)

        u = '''http://player.ku6.com/refer/pdR2iisl-DDqDR-u/v.swf'''
        result = swf(u)
        wanted = '''http://player.ku6.com/refer/pdR2iisl-DDqDR-u/v.swf'''

        self.assertEqual(result, wanted)

        u = '''http://v.game.sohu.com/playercs2008.swf?Flvid='''
        result = swf(u)
        wanted = '''http://v.game.sohu.com/playercs2008.swf?Flvid='''

        self.assertEqual(result, wanted)

        u = '''http://img1.cache.netease.com/flvplayer081128/~false~0085_V6FN25FFF~vimg2.ws.126.net/image/snapshot/2010/9/F/G/V6FN25FFG~.swf&blabla'''
        result = swf(u)
        wanted = '''http://img1.cache.netease.com/flvplayer081128/~false~0085_V6FN25FFF~vimg2.ws.126.net/image/snapshot/2010/9/F/G/V6FN25FFG~.swf&blabla'''

        self.assertEqual(result, wanted)


        u = '''http://swf.ws.126.net/movieplayer/-0-2_M7BN09LGP_M7BN7GOJ7-vimg1_ws_126_net//image/snapshot_movie/2011/8/I/U/M7J1CD9IU-.swf'''
        result = swf(u)
        wanted = '''http://swf.ws.126.net/movieplayer/-0-2_M7BN09LGP_M7BN7GOJ7-vimg1_ws_126_net//image/snapshot_movie/2011/8/I/U/M7J1CD9IU-.swf'''

        self.assertEqual(result, wanted)


    def test_html_escape(self):
        raw = '''< >'''
        wanted = '''&lt; &gt;'''
        result = html_escape(raw)
        self.assertEqual(result, wanted)
    
        raw = '''< >'''
        wanted = '''&lt;&nbsp;&gt;'''
        result = html_escape(raw, True)
        self.assertEqual(result, wanted)
    
    def test_anti_xss(self):
        raw = """[image]" onload="alter('hello');"[/image]"""

        #wanted = """<img src="%22%20onload=%22alter(%27hello%27);%22" />"""
        wanted = u"""<img src="%22 onload%3D%22alter%28%27hello%27%29;%22" />"""
        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """\
[image]/gkimage/xxx/xxx/xxx.png[/image]"""
        wanted = u"""<img src="/gkimage/xxx/xxx/xxx.png" />"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """\
[image]/gkimage/xxx/xxx/xxx.png"[/image]"""
        #wanted = """<img src="/gkimage/xxx/xxx/xxx.png%22" />"""
        wanted = u"""<img src="/gkimage/xxx/xxx/xxx.png%22" />"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[image]http://www.guokr.com/gkimage/eu/31/1j/eu311j.png" onload="alert('XSS')[/image]"""
        wanted = u"""<img src="http://www.guokr.com/gkimage/eu/31/1j/eu311j.png%22 onload%3D%22alert%28%27XSS%27%29" />"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

    def test_valid_color1(self):

        raw = '[color=#00ccff]严重同感。。。虽然睡觉不会超过1点，AND晚上自修也没有那么多闲事做[/color]'
        wanted = u'<span style="color:#00ccff;">严重同感。。。虽然睡觉不会超过1点，AND晚上自修也没有那么多闲事做</span>'

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

    def test_location(self):

        raw = '[[blockquote]引sdfkb[/blockquote]'
        wanted = u'[<blockquote><div class="cmt-line"></div>引sdfkb<div class="cmt-line"></div></blockquote>'

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

    def _test_ref(self):
        raw = """[ref]http://www.guokr.com/article/12345/[/ref]"""
        wanted = u"""<!--# include virtual="/slice/common/article_item/12345/ref_article/" -->"""

        c = compiler(raw, ref_count=1)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[ref]http://www.guokr.com/post/12345/[/ref]"""
        wanted = u"""<!--# include virtual="/slice/common/article_item/12345/ref_post/" -->"""

        c = compiler(raw, ref_count=1)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[ref]http://www.guokr.com/post/12345[/ref]"""
        wanted = u"""<!--# include virtual="/slice/common/article_item/12345/ref_post/" -->"""

        c = compiler(raw, ref_count=1)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[ref]http://www.guokr.com/zone/guokrfact_2011/article/78182/[/ref]"""
        wanted = u"""<!--# include virtual="/slice/common/article_item/78182/ref_zone/" -->"""

        c = compiler(raw, ref_count=2)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[ref]http://www.guokr.com/post/12345[/ref]"""
        wanted = u"""<a href="http://www.guokr.com/post/12345" target="_blank">http://www.guokr.com/post/12345</a>"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[]http://www.guokr.com/post/12345[/ref][ref]http://www.guokr.com/post/12345[/ref]"""
        wanted = u"""[]<a href="http://www.guokr.com/post/12345" target="_blank">http://www.guokr.com/post/12345</a><!--# include virtual="/slice/common/article_item/12345/ref_post/" -->"""

        c = compiler(raw, ref_count=1)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[]http://www.guokr.com/post/12345[/ref][ref]http://www.guokr.com/post/12345[/ref][ref]http://www.guokr.com/post/12345[/ref]"""
        wanted = u"""[]<a href="http://www.guokr.com/post/12345" target="_blank">http://www.guokr.com/post/12345</a><!--# include virtual="/slice/common/article_item/12345/ref_post/" -->[ref]http://www.guokr.com/post/12345[/ref]"""

        c = compiler(raw, ref_count=1)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

    def _test_ul_and_ol(self):
        raw = """[ul]1\n2\n3\n[/ul]"""
        wanted = u"""<ul class="gui-ubbv-ul"><li>1</li><li>2</li><li>3</li></ul>"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[ul]\n1\n2\n3\n[/ul]"""
        wanted = u"""<ul class="gui-ubbv-ul"><li>1</li><li>2</li><li>3</li></ul>"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[ol]1\n2\n3\n[/ol]"""
        wanted = u"""<ol class="gui-ubbv-ol"><li>1</li><li>2</li><li>3</li></ol>"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[ol]1\n\n\n\n\n\n2\n3\n\n\n\n\n\n\n\n\n\n\n4[/ol]"""
        wanted = u"""<ol class="gui-ubbv-ol"><li>1</li><li>2</li><li>3</li><li>4</li></ol>"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[blockquote][ol]1\n2\n3\n[/ol][/blockquote]"""
        wanted = u"""<blockquote><div class="cmt-line"></div><ol class="gui-ubbv-ol"><li>1</li><li>2</li><li>3</li></ol><div class="cmt-line"></div></blockquote>"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

        raw = """[ul]1\n2\n3\n[/ul]\r\n[ul]1\n2\n3\n[/ul]\r\nblabla"""
        wanted = u"""<ul class="gui-ubbv-ul"><li>1</li><li>2</li><li>3</li></ul><ul class="gui-ubbv-ul"><li>1</li><li>2</li><li>3</li></ul>\rblabla"""

        c = compiler(raw)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)
        
    def test_li_with_other(self):
        raw = """[ul]1\n[ref]www.guokr.com/post/12345/[/ref]\n3\n[/ul]"""
        wanted = u"""<ul class="gui-ubbv-ul"><li>1</li><li><!--# include virtual="/slice/common/article_item/12345/ref_post/" --></li><li>3</li></ul>"""

        c = compiler(raw, ref_count = 1)
        result = c.ubbtohtml()
        self.assertEqual(result, wanted)

    def test_mark(self):
        raw = """1[bold]2[/BOLD]3[italic]4[/italic]!"""
        wanted = [(u'1', None), (u'2', 'bold'), (u'3', None), (u'4', 'italic'), (u'!', None)]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

        raw = """1[bold]23[italic]4[/italic][/bold]!"""
        wanted = [(u'1', None), (((u'23', None), (u'4', 'italic')), 'bold'), (u'!', None)]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

        raw = """[image]http://www.guokr.com/abc.png[/image]"""
        wanted = [(u'http://www.guokr.com/abc.png', 'image')]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

        raw = """[ul]\n1\n[image]http://www.guokr.com/abc.png[/image]\n3\n[/ul]"""
        wanted = [(((u'\n1\n', None), (u'http://www.guokr.com/abc.png', 'image'), (u'\n3\n', None)), 'ul')]

        raw = """1[blockquote]2[/blockquote]3[italic]4[/italic]!"""
        wanted = [(u'1', None), (u'2', 'blockquote'), (u'3', None), (u'4', 'italic'), (u'!', None)]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

        raw = """http://www.guokr.com"""
        wanted = [(u'http://www.guokr.com', 'url')]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

        raw = """[ul][/ul]"""
        wanted = [((), 'ul')]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)
        
        raw = """1[bold][italic]4[/italic][/bold]!"""
        wanted = [(u'1', None), (((u'4', 'italic'), ), 'bold'), (u'!', None)]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

        raw = """
[ul]
[bold]内容1[/bold]内容2
内容3
内容4
[/ul]"""
        wanted = [(u'\n', None), ((((u'内容1', 'bold'), (u'内容2', None)), ((u'内容3', None), ), ((u'内容4', None), )), 'ul')]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

        raw = """"""
        wanted = []

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

        raw = """[url=http://www.guokr.com][image]www.guokr.com[/image][/url]"""
        wanted = [(((u'www.guokr.com', 'image'), ), 'url')]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

        raw = """
[ol]
其次，认同@[url href=http://www.guokr.com/i/0173548288/]程旸[/url]DEF
ABC
[/ol]
"""
        wanted = [(u'\n', None), ((((u'其次，认同@', None),(u'程旸', 'url'), (u'DEF', None)), ((u'ABC', None), )), 'ol'), (u'\n', None)]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

        raw = """[url href=http://huangx.in]http://huangx.in[/url]
[ul]
[image]/gkimage/st/g5/uu/stg5uu.png[/image]123
好内容1
好内容2
[bold]坏内容[/bold]
[/ul]
"""
        wanted = [(u'http://huangx.in', 'url'), (u'\n', None), ((((u'/gkimage/st/g5/uu/stg5uu.png', 'image'), (u'123', None)), ((u'好内容1', None), ), ((u'好内容2', None), ), ((u'坏内容', 'bold'), )), 'ul'), (u'\n', None)]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

        raw = """
[ul]
[url href=http://www.guokr.com][image]/gkimage/kj/yj/0a/kjyj0a.png[/image][/url]
[/ul]
"""
        wanted = [(((((u'/gkimage/kj/yj/0a/kjyj0a.png', 'image'), ), 'url'), ), 'ul'), (u'\n', None)]

        c = compiler(raw)
        result = c.mark()
#        self.assertEqual(result, wanted) TODO 嵌套问题过不去 修改量太大 果壳网内暂时也无对应数据 留待重构

        raw = """不换行？
[url href=http://huangx.in]http://huangx.in[/url]
[url href=http://huangx.in]http://huangx.in[/url]
[image]/gkimage/st/g5/uu/stg5uu.png[/image]123
好内容1
好内容2
[bold]坏内容[/bold]
"""
        wanted = [(u'不换行？\n', None), (u'http://huangx.in', 'url'), (u'\n', None), (u'http://huangx.in', 'url'), (u'\n', None), (u'/gkimage/st/g5/uu/stg5uu.png', 'image'), (u'123\n好内容1\n好内容2\n', None), (u'坏内容', 'bold'), (u'\n', None)]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)

    def test_wrong(self):
        raw = """[bold]123 http://123.com [italic]789[/italic][/bold]456"""
        wanted = [(((u'123 ', None), (u'http://123.com', 'url'), (' ', None), (u'789', 'italic')), 'bold'), (u'456', None)]

        c = compiler(raw)
        result = c.mark()
        self.assertEqual(result, wanted)


if '__main__' == __name__:
        unittest.main()
